var fs = require('fs');

var washText = function (text) {
  var textWashed = text.toLowerCase();
  textWashed = textWashed.replace(/[^a-z]+/g, ' ');
  return textWashed;
}

var extractWords = function (text) {
  var wordDict = {};

  var wordsInText = text.split(' ');
  
  for (var i=0; i<wordsInText.length; ++i) {
    var w = wordsInText[i];
    if (!wordDict[w]) {
      wordDict[w] = 1;
    } else {
      wordDict[w]++;
    }
  }

  return wordDict;
}

var computeWordFreq = function (wordDict) {
  var wordFreq = [];

  for (var w in wordDict) {
    wordFreq.push({
      'word': w,
      'freq': wordDict[w]
    });
  }

  wordFreq.sort(function (w1, w2) {
    if (w1.freq < w2.freq) return 1;
    if (w1.freq > w2.freq) return -1;
    return 0;
  });

  return wordFreq;
}

var computeWordLengthFreqDict = function (wordDict) {
  var wordLengthFreqDict = {};

  for (var w in wordDict) {
    if (!wordLengthFreqDict[w.length]) {
      wordLengthFreqDict[w.length] = {
        'nWords': 1,
        'freqSum': wordDict[w],
        'freq': wordDict[w]
      };
    } else {
      wordLengthFreqDict[w.length]['nWords']++;
      wordLengthFreqDict[w.length]['freqSum'] += wordDict[w];
      wordLengthFreqDict[w.length]['freq'] = 
        wordLengthFreqDict[w.length]['freqSum'] / wordLengthFreqDict[w.length]['nWords'];
    }
  }

  return wordLengthFreqDict;
}

var computeWordLengthFreq = function (wordDict) {
  var wordLengthFreq = [];

  var wordLengthFreqDict = computeWordLengthFreqDict(wordDict);

  for (var wl in wordLengthFreqDict) {
    wordLengthFreq.push({
      'wordLength': wl,
      'nWords': wordLengthFreqDict[wl]['nWords'],
      'freqSum': wordLengthFreqDict[wl]['freqSum'],
      'freq': wordLengthFreqDict[wl]['freq']
    });
  }

  wordLengthFreq.sort(function (wl1, wl2) {
    if (wl1.freq < wl2.freq) return 1;
    if (wl1.freq > wl2.freq) return -1;
    return 0;
  });

  return wordLengthFreq;
}

var refineWordDict = function (wordDict) {
  var PARAM_WORD_LENGTH_CRITERIUM = 0.05; // < 0.001 for large data set, > 0.05 for small data set

  var refinedWordDict = {};

  for (var w in wordDict) {
    if (w.length < 2 && w != 'a') continue;
    if (wordDict[w] < 2) continue;

    refinedWordDict[w] = wordDict[w];
  }

  wordDict = refinedWordDict;
  refinedWordDict = {};

  var wordLengthFreqDict = computeWordLengthFreqDict(wordDict);

  for (var w in wordDict) {
    if (wordDict[w] < wordLengthFreqDict[w.length]['freq'] * PARAM_WORD_LENGTH_CRITERIUM) continue;

    refinedWordDict[w] = wordDict[w];
  }

  return refinedWordDict;
}

fs.readFile('sample_raw.txt', 'utf8', function (err, data) {
  if (err) throw err;

  var text = washText(data);

  var wordDict = extractWords(text);

  var wordDict = refineWordDict(wordDict);

  var wordFreq = computeWordFreq(wordDict);

  var wordLengthFreq = computeWordLengthFreq(wordDict);

  console.log(wordFreq);
  //console.log(wordLengthFreq);
});

